import requests
from bs4 import BeautifulSoup
import multiprocessing
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

headers = {
    'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
    'cookie' : '__qhsrc=0; ckmts=PUUt8Sh_,L64t8Sh_,J6bdEH0_; v=BP+mN96SyT:WUqd-<57`',
}
def get_link(url):
    print('获取书的父进程为：{}'.format(multiprocessing.current_process().pid))
    html = requests.get(url, headers = headers)
    soup = BeautifulSoup(html.text, 'lxml')
    title = soup.select_one('#wrapper h1 span').text
    print(title)



def get_html(url):
    print('html当前进程{}'.format(multiprocessing.current_process().pid))
    html = requests.get(url, headers = headers)
    threadpools = ThreadPoolExecutor(max_workers=3)
    soup = BeautifulSoup(html.text, 'lxml')
    urls = soup.select('.subject-item .info h2 a')
    for url in urls:
        link = url['href']
        threadpools.submit(get_link, link)


if __name__ == '__main__':
    urls=[
        'https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4',
        'https://book.douban.com/tag/%E5%90%8D%E8%91%97',
        'https://book.douban.com/tag/%E7%AB%A5%E8%AF%9D',
    ]
    with ProcessPoolExecutor(max_workers=3)as executor:
        futurres =  [executor.submit(get_html, url) for url in urls]